package cn.dmp.tools

import cn.dmp.utils.JedisPools
import org.apache.spark.{SparkConf, SparkContext}

/**
  * 将app字典数据（app知识库）导入到redis中
  */
object AppDict2Redis {

    def main(args: Array[String]): Unit = {

        val sparkConf = new SparkConf()
          .setMaster("local[*]")
          .setAppName("appdict2redis")
          .set("spark.serializer", "org.apache.spark.serializer.KryoSerializer") // RDD | worker

        val sc = new SparkContext(sparkConf)
        // 接受参数
        val Array(dataInputPath) = args

        // 读取字典文件
        val filtered = sc.textFile(dataInputPath).map(_.split("\t", -1)).filter(_.length >=5)
        // 将字典文件中的appid ,appname 存到redis中
        filtered.foreachPartition(itr => {
            val jedis = JedisPools.getJedis
            itr.map(arr => (arr(4), arr(1))).foreach(t => {
                jedis.hset("appdict", t._1, t._2)
            })
            jedis.close()
        })
        sc.stop()
    }

}
